import requests
import time
import json
import os
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
import logging

class ThreeJSDocsScraper:
    def __init__(self, base_url="http://www.yanhuangxueyuan.com/threejs/docs/", max_workers=5):
        self.base_url = base_url
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Accept-Encoding': 'gzip, deflate',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1'
        })
        self.max_workers = max_workers
        self.scraped_urls = set()
        self.failed_urls = []
        
        # 设置日志
        logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
        self.logger = logging.getLogger(__name__)
    
    def get_page_content(self, url, retries=3):
        """获取页面内容"""
        for attempt in range(retries):
            try:
                response = self.session.get(url, timeout=30)
                response.raise_for_status()
                response.encoding = 'utf-8'
                return response.text
            except requests.RequestException as e:
                self.logger.warning(f"获取页面失败 (尝试 {attempt + 1}/{retries}): {url} - {str(e)}")
                if attempt < retries - 1:
                    time.sleep(2 ** attempt)  # 指数退避
                else:
                    self.failed_urls.append(url)
                    return None
    
    def parse_main_page(self):
        """解析主页面，获取所有文档链接"""
        main_url = urljoin(self.base_url, "index.html")
        self.logger.info(f"正在获取页面内容: {main_url}")
        content = self.get_page_content(main_url)
        
        if not content:
            self.logger.error("无法获取主页面内容")
            return []
        
        self.logger.info(f"成功获取页面内容，长度: {len(content)} 字符")
        soup = BeautifulSoup(content, 'html.parser')
        self.logger.info(f"页面标题: {soup.title.text if soup.title else '无标题'}")
        doc_links = []        # 查找所有的文档链接
        # 基于你提供的文档结构，查找所有相关链接
        link_patterns = [
            r'#api/zh/.*',  # API文档链接 - 锚点形式
            r'api/zh/.*\.html',  # API文档链接 - 直接HTML文件
            r'manual/zh/.*'  # 手册链接
        ]
        
        for link in soup.find_all('a', href=True):
            href = link['href']
            for pattern in link_patterns:
                if re.search(pattern, href):  # 使用re.search代替re.match来匹配任意位置
                    full_url = urljoin(main_url, href)
                    doc_links.append({
                        'url': full_url,
                        'title': link.get_text(strip=True),
                        'category': self.categorize_link(href)
                    })
        
        # 手动添加一些重要的API分类链接
        api_categories = [
            'animation', 'audio', 'cameras', 'constants', 'core', 'extras', 
            'geometries', 'helpers', 'lights', 'loaders', 'materials', 
            'math', 'objects', 'renderers', 'scenes', 'textures'
        ]
        
        for category in api_categories:
            category_url = f"{self.base_url}index.html#api/zh/{category}"
            doc_links.append({
                'url': category_url,
                'title': category.title(),
                'category': category
            })
        
        self.logger.info(f"找到 {len(doc_links)} 个文档链接")
        return doc_links
    
    def categorize_link(self, href):
        """根据链接路径分类"""
        # 主要分类
        main_categories = {
            'animation': 'animation',
            'audio': 'audio',
            'camera': 'cameras',
            'core': 'core',
            'geometr': 'geometries',
            'material': 'materials',
            'math': 'math',
            'light': 'lights',
            'loader': 'loaders',
            'renderer': 'renderers',
            'texture': 'textures',
            'scene': 'scenes',
            'object': 'objects',
            'helper': 'helpers',
            'constant': 'constants',
            'extra': 'extras',
            'manual': 'manual',
            'example': 'examples'
        }
        
        # 特定子分类路径处理
        if '/tracks/' in href:
            return 'animation/tracks'
        elif '/shadows/' in href:
            return 'lights/shadows'
        elif '/managers/' in href:
            return 'loaders/managers'
        elif '/shaders/' in href:
            return 'renderers/shaders'
        elif '/webxr/' in href:
            return 'renderers/webxr'
        elif '/interpolants/' in href:
            return 'math/interpolants'
        elif '/bufferAttributeTypes/' in href:
            return 'core/bufferAttributeTypes'
        elif '/curves/' in href:
            return 'extras/curves'
        elif '/core/' in href and 'extras' in href:
            return 'extras/core'
        elif '/controls/' in href:
            return 'addons/controls'
        elif '/utils/' in href and 'examples' in href:
            return 'addons/utils'
        elif '/postprocessing/' in href:
            return 'addons/postprocessing'
            
        # 一般分类处理
        for key, category in main_categories.items():
            if key in href:
                return category
                
        return 'other'
    
    def parse_doc_page(self, doc_info):
        """解析单个文档页面"""
        url = doc_info['url']
        
        if url in self.scraped_urls:
            return None
        
        self.scraped_urls.add(url)
        content = self.get_page_content(url)
        
        if not content:
            return None
        
        soup = BeautifulSoup(content, 'html.parser')
        
        # 提取文档内容
        doc_data = {
            'url': url,
            'title': doc_info.get('title', ''),
            'category': doc_info.get('category', ''),
            'content': '',
            'methods': [],
            'properties': [],
            'examples': [],
            'description': ''
        }
        
        # 提取主要内容区域
        main_content = soup.find('div', class_='content') or soup.find('main') or soup.body
        
        if main_content:
            # 提取标题
            title_elem = main_content.find(['h1', 'h2']) 
            if title_elem and not doc_data['title']:
                doc_data['title'] = title_elem.get_text(strip=True)
            
            # 提取描述
            desc_elem = main_content.find('p')
            if desc_elem:
                doc_data['description'] = desc_elem.get_text(strip=True)
            
            # 提取方法和属性
            for header in main_content.find_all(['h2', 'h3', 'h4']):
                header_text = header.get_text(strip=True).lower()
                next_content = self.get_next_content(header)
                
                if '方法' in header_text or 'method' in header_text:
                    doc_data['methods'].extend(self.extract_methods(next_content))
                elif '属性' in header_text or 'propert' in header_text:
                    doc_data['properties'].extend(self.extract_properties(next_content))
                elif '示例' in header_text or 'example' in header_text:
                    doc_data['examples'].extend(self.extract_examples(next_content))
            
            # 提取完整文本内容
            doc_data['content'] = main_content.get_text(separator='\n', strip=True)
        
        self.logger.info(f"成功解析: {doc_data['title']} ({url})")
        return doc_data
    
    def get_next_content(self, header):
        """获取标题后的内容直到下一个同级标题"""
        content = []
        current = header.next_sibling
        
        while current:
            if current.name and current.name.startswith('h') and len(current.name) <= len(header.name):
                break
            if hasattr(current, 'get_text'):
                content.append(current)
            current = current.next_sibling
        
        return content
    
    def extract_methods(self, content_elements):
        """提取方法信息"""
        methods = []
        for elem in content_elements:
            if hasattr(elem, 'find_all'):
                # 查找代码块或方法定义
                code_blocks = elem.find_all(['code', 'pre'])
                for code in code_blocks:
                    method_text = code.get_text(strip=True)
                    if '(' in method_text and ')' in method_text:
                        methods.append(method_text)
        return methods
    
    def extract_properties(self, content_elements):
        """提取属性信息"""
        properties = []
        for elem in content_elements:
            if hasattr(elem, 'find_all'):
                # 查找属性定义
                prop_items = elem.find_all(['li', 'dt', 'code'])
                for item in prop_items:
                    prop_text = item.get_text(strip=True)
                    if prop_text and not prop_text.startswith('('):
                        properties.append(prop_text)
        return properties
    
    def extract_examples(self, content_elements):
        """提取示例代码"""
        examples = []
        for elem in content_elements:
            if hasattr(elem, 'find_all'):
                code_blocks = elem.find_all(['pre', 'code'])
                for code in code_blocks:
                    example_text = code.get_text(strip=True)
                    if len(example_text) > 20:  # 过滤掉太短的代码片段
                        examples.append(example_text)
        return examples
    
    def scrape_single_doc(self, doc_info):
        """爬取单个文档的包装函数"""
        try:
            return self.parse_doc_page(doc_info)
        except Exception as e:
            self.logger.error(f"解析文档时出错: {doc_info['url']} - {str(e)}")
            return None
    
    def scrape_all_docs(self, output_dir="threejs_docs"):
        """爬取所有文档"""
        # 创建输出目录
        os.makedirs(output_dir, exist_ok=True)
        
        # 获取所有文档链接
        self.logger.info("开始获取文档链接...")
        doc_links = self.parse_main_page()
        
        if not doc_links:
            self.logger.error("未找到任何文档链接")
            return
        
        # 使用线程池并发爬取
        all_docs = []
        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            future_to_doc = {executor.submit(self.scrape_single_doc, doc): doc for doc in doc_links}
            
            for future in as_completed(future_to_doc):
                doc_data = future.result()
                if doc_data:
                    all_docs.append(doc_data)
                
                # 避免请求过于频繁
                time.sleep(0.5)
        
        # 按分类保存文档
        docs_by_category = {}
        for doc in all_docs:
            category = doc['category']
            if category not in docs_by_category:
                docs_by_category[category] = []
            docs_by_category[category].append(doc)
        
        # 保存结果
        for category, docs in docs_by_category.items():
            category_file = os.path.join(output_dir, f"{category}.json")
            with open(category_file, 'w', encoding='utf-8') as f:
                json.dump(docs, f, ensure_ascii=False, indent=2)
            self.logger.info(f"保存 {category} 分类文档: {len(docs)} 个")
        
        # 保存完整数据
        all_docs_file = os.path.join(output_dir, "all_docs.json")
        with open(all_docs_file, 'w', encoding='utf-8') as f:
            json.dump(all_docs, f, ensure_ascii=False, indent=2)
        
        # 保存统计信息
        stats = {
            'total_docs': len(all_docs),
            'categories': list(docs_by_category.keys()),
            'failed_urls': self.failed_urls,
            'scrape_time': time.strftime('%Y-%m-%d %H:%M:%S')
        }
        
        stats_file = os.path.join(output_dir, "scrape_stats.json")
        with open(stats_file, 'w', encoding='utf-8') as f:
            json.dump(stats, f, ensure_ascii=False, indent=2)
        
        self.logger.info(f"爬取完成! 总共获取 {len(all_docs)} 个文档")
        self.logger.info(f"失败的URL: {len(self.failed_urls)} 个")
        
        return all_docs

    def process_html_content(self, html_content, base_category="animation"):
        """处理直接提供的HTML内容，从中提取文档链接"""
        soup = BeautifulSoup(html_content, 'html.parser')
        doc_links = []
        
        # 查找所有链接
        for link in soup.find_all('a', href=True):
            href = link['href']
            if 'api/zh/' in href or 'manual/zh/' in href:
                # 构建完整URL
                full_url = urljoin(self.base_url, href)
                
                # 提取分类 (从链接路径中)
                category = self.categorize_link(href)
                if not category or category == 'other':
                    category = base_category
                
                doc_links.append({
                    'url': full_url,
                    'title': link.get_text(strip=True),
                    'category': category
                })
                self.logger.info(f"添加文档链接: {full_url} ({link.get_text(strip=True)})")
        
        return doc_links

def main():
    """主函数"""
    scraper = ThreeJSDocsScraper(max_workers=3)  # 减少并发数避免被封
    
    try:
        # 1. 尝试常规爬取
        print("尝试常规爬取...")
        docs = scraper.scrape_all_docs("threejs_docs_output")
        
        # 2. 如果没有找到文档，尝试使用提供的HTML内容
        if not docs or len(docs) == 0:
            print("常规爬取未找到文档，尝试使用提供的HTML内容...")
            # 这里是你提供的HTML片段
            animation_html = """
            <div><h2>手册</h2><div><h3>起步</h3><ul><li><a href="manual/zh/introduction/Installation.html" target="viewer">安装</a></li><li><a href="manual/zh/introduction/Creating-a-scene.html" target="viewer">创建一个场景</a></li><li><a href="manual/zh/introduction/WebGL-compatibility-check.html" target="viewer">WebGL兼容性检查</a></li><li><a href="manual/zh/introduction/Drawing-lines.html" target="viewer">画线</a></li><li><a href="manual/zh/introduction/Creating-text.html" target="viewer">创建文字</a></li><li><a href="manual/zh/introduction/Loading-3D-models.html" target="viewer">载入3D模型</a></li><li><a href="manual/zh/introduction/FAQ.html" target="viewer">常见问题</a></li><li><a href="manual/zh/introduction/Useful-links.html" target="viewer">一些有用的链接</a></li></ul></div><div><h3>进阶</h3><ul><li><a href="manual/zh/introduction/How-to-update-things.html" target="viewer">如何更新场景</a></li><li><a href="manual/zh/introduction/How-to-dispose-of-objects.html" target="viewer">如何废置对象</a></li><li><a href="manual/zh/introduction/How-to-create-VR-content.html" target="viewer">如何创建VR内容</a></li><li><a href="manual/zh/introduction/How-to-use-post-processing.html" target="viewer">如何使用后期处理</a></li><li><a href="manual/zh/introduction/Matrix-transformations.html" target="viewer">矩阵变换</a></li><li><a href="manual/zh/introduction/Animation-system.html" target="viewer">动画系统</a></li></ul></div><h2>参考</h2><div><h3>动画</h3><ul><li><a href="api/zh/animation/AnimationAction.html" target="viewer" class="selected">AnimationAction</a></li><li><a href="api/zh/animation/AnimationClip.html" target="viewer" class="">AnimationClip</a></li><li><a href="api/zh/animation/AnimationMixer.html" target="viewer">AnimationMixer</a></li><li><a href="api/zh/animation/AnimationObjectGroup.html" target="viewer">AnimationObjectGroup</a></li><li><a href="api/zh/animation/AnimationUtils.html" target="viewer">AnimationUtils</a></li><li><a href="api/zh/animation/KeyframeTrack.html" target="viewer">KeyframeTrack</a></li><li><a href="api/zh/animation/PropertyBinding.html" target="viewer">PropertyBinding</a></li><li><a href="api/zh/animation/PropertyMixer.html" target="viewer">PropertyMixer</a></li></ul></div><div><h3>动画 / 轨道</h3><ul><li><a href="api/zh/animation/tracks/BooleanKeyframeTrack.html" target="viewer">BooleanKeyframeTrack</a></li><li><a href="api/zh/animation/tracks/ColorKeyframeTrack.html" target="viewer">ColorKeyframeTrack</a></li><li><a href="api/zh/animation/tracks/NumberKeyframeTrack.html" target="viewer">NumberKeyframeTrack</a></li><li><a href="api/zh/animation/tracks/QuaternionKeyframeTrack.html" target="viewer">QuaternionKeyframeTrack</a></li><li><a href="api/zh/animation/tracks/StringKeyframeTrack.html" target="viewer">StringKeyframeTrack</a></li><li><a href="api/zh/animation/tracks/VectorKeyframeTrack.html" target="viewer">VectorKeyframeTrack</a></li></ul></div><div><h3>音频</h3><ul><li><a href="api/zh/audio/Audio.html" target="viewer">Audio</a></li><li><a href="api/zh/audio/AudioAnalyser.html" target="viewer">AudioAnalyser</a></li><li><a href="api/zh/audio/AudioContext.html" target="viewer">AudioContext</a></li><li><a href="api/zh/audio/AudioListener.html" target="viewer">AudioListener</a></li><li><a href="api/zh/audio/PositionalAudio.html" target="viewer">PositionalAudio</a></li></ul></div><div><h3>摄像机</h3><ul><li><a href="api/zh/cameras/ArrayCamera.html" target="viewer">ArrayCamera</a></li><li><a href="api/zh/cameras/Camera.html" target="viewer">Camera</a></li><li><a href="api/zh/cameras/CubeCamera.html" target="viewer">CubeCamera</a></li><li><a href="api/zh/cameras/OrthographicCamera.html" target="viewer">OrthographicCamera</a></li><li><a href="api/zh/cameras/PerspectiveCamera.html" target="viewer">PerspectiveCamera</a></li><li><a href="api/zh/cameras/StereoCamera.html" target="viewer">StereoCamera</a></li></ul></div><div><h3>常量</h3><ul><li><a href="api/zh/constants/Animation.html" target="viewer">Animation</a></li><li><a href="api/zh/constants/Core.html" target="viewer">Core</a></li><li><a href="api/zh/constants/CustomBlendingEquations.html" target="viewer">CustomBlendingEquation</a></li><li><a href="api/zh/constants/Materials.html" target="viewer">Materials</a></li><li><a href="api/zh/constants/Renderer.html" target="viewer">Renderer</a></li><li><a href="api/zh/constants/Textures.html" target="viewer">Textures</a></li></ul></div><div><h3>核心</h3><ul><li><a href="api/zh/core/BufferAttribute.html" target="viewer">BufferAttribute</a></li><li><a href="api/zh/core/BufferGeometry.html" target="viewer">BufferGeometry</a></li><li><a href="api/zh/core/Clock.html" target="viewer">Clock</a></li><li><a href="api/zh/core/EventDispatcher.html" target="viewer">EventDispatcher</a></li><li><a href="api/zh/core/GLBufferAttribute.html" target="viewer">GLBufferAttribute</a></li><li><a href="api/zh/core/InstancedBufferAttribute.html" target="viewer">InstancedBufferAttribute</a></li><li><a href="api/zh/core/InstancedBufferGeometry.html" target="viewer">InstancedBufferGeometry</a></li><li><a href="api/zh/core/InstancedInterleavedBuffer.html" target="viewer">InstancedInterleavedBuffer</a></li><li><a href="api/zh/core/InterleavedBuffer.html" target="viewer">InterleavedBuffer</a></li><li><a href="api/zh/core/InterleavedBufferAttribute.html" target="viewer">InterleavedBufferAttribute</a></li><li><a href="api/zh/core/Layers.html" target="viewer">Layers</a></li><li><a href="api/zh/core/Object3D.html" target="viewer">Object3D</a></li><li><a href="api/zh/core/Raycaster.html" target="viewer">Raycaster</a></li><li><a href="api/zh/core/Uniform.html" target="viewer">Uniform</a></li></ul></div><div><h3>核心 / BufferAttributes</h3><ul><li><a href="api/zh/core/bufferAttributeTypes/BufferAttributeTypes.html" target="viewer">BufferAttribute Types</a></li></ul></div><div><h3>附件</h3><ul><li><a href="api/zh/extras/DataUtils.html" target="viewer">DataUtils</a></li><li><a href="api/zh/extras/Earcut.html" target="viewer">Earcut</a></li><li><a href="api/zh/extras/ImageUtils.html" target="viewer">ImageUtils</a></li><li><a href="api/zh/extras/PMREMGenerator.html" target="viewer">PMREMGenerator</a></li><li><a href="api/zh/extras/ShapeUtils.html" target="viewer">ShapeUtils</a></li></ul></div><div><h3>附件 / 核心</h3><ul><li><a href="api/zh/extras/core/Curve.html" target="viewer">Curve</a></li><li><a href="api/zh/extras/core/CurvePath.html" target="viewer">CurvePath</a></li><li><a href="api/zh/extras/core/Interpolations.html" target="viewer">Interpolations</a></li><li><a href="api/zh/extras/core/Path.html" target="viewer">Path</a></li><li><a href="api/zh/extras/core/Shape.html" target="viewer">Shape</a></li><li><a href="api/zh/extras/core/ShapePath.html" target="viewer">ShapePath</a></li></ul></div><div><h3>附件 / 曲线</h3><ul><li><a href="api/zh/extras/curves/ArcCurve.html" target="viewer">ArcCurve</a></li><li><a href="api/zh/extras/curves/CatmullRomCurve3.html" target="viewer">CatmullRomCurve3</a></li><li><a href="api/zh/extras/curves/CubicBezierCurve.html" target="viewer">CubicBezierCurve</a></li><li><a href="api/zh/extras/curves/CubicBezierCurve3.html" target="viewer">CubicBezierCurve3</a></li><li><a href="api/zh/extras/curves/EllipseCurve.html" target="viewer">EllipseCurve</a></li><li><a href="api/zh/extras/curves/LineCurve.html" target="viewer">LineCurve</a></li><li><a href="api/zh/extras/curves/LineCurve3.html" target="viewer">LineCurve3</a></li><li><a href="api/zh/extras/curves/QuadraticBezierCurve.html" target="viewer">QuadraticBezierCurve</a></li><li><a href="api/zh/extras/curves/QuadraticBezierCurve3.html" target="viewer">QuadraticBezierCurve3</a></li><li><a href="api/zh/extras/curves/SplineCurve.html" target="viewer">SplineCurve</a></li></ul></div><div><h3>几何体</h3><ul><li><a href="api/zh/geometries/BoxGeometry.html" target="viewer">BoxGeometry</a></li><li><a href="api/zh/geometries/CapsuleGeometry.html" target="viewer">CapsuleGeometry</a></li><li><a href="api/zh/geometries/CircleGeometry.html" target="viewer">CircleGeometry</a></li><li><a href="api/zh/geometries/ConeGeometry.html" target="viewer">ConeGeometry</a></li><li><a href="api/zh/geometries/CylinderGeometry.html" target="viewer">CylinderGeometry</a></li><li><a href="api/zh/geometries/DodecahedronGeometry.html" target="viewer">DodecahedronGeometry</a></li><li><a href="api/zh/geometries/EdgesGeometry.html" target="viewer">EdgesGeometry</a></li><li><a href="api/zh/geometries/ExtrudeGeometry.html" target="viewer">ExtrudeGeometry</a></li><li><a href="api/zh/geometries/IcosahedronGeometry.html" target="viewer">IcosahedronGeometry</a></li><li><a href="api/zh/geometries/LatheGeometry.html" target="viewer">LatheGeometry</a></li><li><a href="api/zh/geometries/OctahedronGeometry.html" target="viewer">OctahedronGeometry</a></li><li><a href="api/zh/geometries/PlaneGeometry.html" target="viewer">PlaneGeometry</a></li><li><a href="api/zh/geometries/PolyhedronGeometry.html" target="viewer">PolyhedronGeometry</a></li><li><a href="api/zh/geometries/RingGeometry.html" target="viewer">RingGeometry</a></li><li><a href="api/zh/geometries/ShapeGeometry.html" target="viewer">ShapeGeometry</a></li><li><a href="api/zh/geometries/SphereGeometry.html" target="viewer">SphereGeometry</a></li><li><a href="api/zh/geometries/TetrahedronGeometry.html" target="viewer">TetrahedronGeometry</a></li><li><a href="api/zh/geometries/TorusGeometry.html" target="viewer">TorusGeometry</a></li><li><a href="api/zh/geometries/TorusKnotGeometry.html" target="viewer">TorusKnotGeometry</a></li><li><a href="api/zh/geometries/TubeGeometry.html" target="viewer">TubeGeometry</a></li><li><a href="api/zh/geometries/WireframeGeometry.html" target="viewer">WireframeGeometry</a></li></ul></div><div><h3>辅助对象</h3><ul><li><a href="api/zh/helpers/ArrowHelper.html" target="viewer">ArrowHelper</a></li><li><a href="api/zh/helpers/AxesHelper.html" target="viewer">AxesHelper</a></li><li><a href="api/zh/helpers/BoxHelper.html" target="viewer">BoxHelper</a></li><li><a href="api/zh/helpers/Box3Helper.html" target="viewer">Box3Helper</a></li><li><a href="api/zh/helpers/CameraHelper.html" target="viewer">CameraHelper</a></li><li><a href="api/zh/helpers/DirectionalLightHelper.html" target="viewer">DirectionalLightHelper</a></li><li><a href="api/zh/helpers/GridHelper.html" target="viewer">GridHelper</a></li><li><a href="api/zh/helpers/PolarGridHelper.html" target="viewer">PolarGridHelper</a></li><li><a href="api/zh/helpers/HemisphereLightHelper.html" target="viewer">HemisphereLightHelper</a></li><li><a href="api/zh/helpers/PlaneHelper.html" target="viewer">PlaneHelper</a></li><li><a href="api/zh/helpers/PointLightHelper.html" target="viewer">PointLightHelper</a></li><li><a href="api/zh/helpers/SkeletonHelper.html" target="viewer">SkeletonHelper</a></li><li><a href="api/zh/helpers/SpotLightHelper.html" target="viewer">SpotLightHelper</a></li></ul></div><div><h3>灯光</h3><ul><li><a href="api/zh/lights/AmbientLight.html" target="viewer">AmbientLight</a></li><li><a href="api/zh/lights/AmbientLightProbe.html" target="viewer">AmbientLightProbe</a></li><li><a href="api/zh/lights/DirectionalLight.html" target="viewer">DirectionalLight</a></li><li><a href="api/zh/lights/HemisphereLight.html" target="viewer">HemisphereLight</a></li><li><a href="api/zh/lights/HemisphereLightProbe.html" target="viewer">HemisphereLightProbe</a></li><li><a href="api/zh/lights/Light.html" target="viewer">Light</a></li><li><a href="api/zh/lights/LightProbe.html" target="viewer">LightProbe</a></li><li><a href="api/zh/lights/PointLight.html" target="viewer">PointLight</a></li><li><a href="api/zh/lights/RectAreaLight.html" target="viewer">RectAreaLight</a></li><li><a href="api/zh/lights/SpotLight.html" target="viewer">SpotLight</a></li></ul></div><div><h3>灯光 / 阴影</h3><ul><li><a href="api/zh/lights/shadows/LightShadow.html" target="viewer">LightShadow</a></li><li><a href="api/zh/lights/shadows/PointLightShadow.html" target="viewer">PointLightShadow</a></li><li><a href="api/zh/lights/shadows/DirectionalLightShadow.html" target="viewer">DirectionalLightShadow</a></li><li><a href="api/zh/lights/shadows/SpotLightShadow.html" target="viewer">SpotLightShadow</a></li></ul></div><div><h3>加载器</h3><ul><li><a href="api/zh/loaders/AnimationLoader.html" target="viewer">AnimationLoader</a></li><li><a href="api/zh/loaders/AudioLoader.html" target="viewer">AudioLoader</a></li><li><a href="api/zh/loaders/BufferGeometryLoader.html" target="viewer">BufferGeometryLoader</a></li><li><a href="api/zh/loaders/Cache.html" target="viewer">Cache</a></li><li><a href="api/zh/loaders/CompressedTextureLoader.html" target="viewer">CompressedTextureLoader</a></li><li><a href="api/zh/loaders/CubeTextureLoader.html" target="viewer">CubeTextureLoader</a></li><li><a href="api/zh/loaders/DataTextureLoader.html" target="viewer">DataTextureLoader</a></li><li><a href="api/zh/loaders/FileLoader.html" target="viewer">FileLoader</a></li><li><a href="api/zh/loaders/ImageBitmapLoader.html" target="viewer">ImageBitmapLoader</a></li><li><a href="api/zh/loaders/ImageLoader.html" target="viewer">ImageLoader</a></li><li><a href="api/zh/loaders/Loader.html" target="viewer">Loader</a></li><li><a href="api/zh/loaders/LoaderUtils.html" target="viewer">LoaderUtils</a></li><li><a href="api/zh/loaders/MaterialLoader.html" target="viewer">MaterialLoader</a></li><li><a href="api/zh/loaders/ObjectLoader.html" target="viewer">ObjectLoader</a></li><li><a href="api/zh/loaders/TextureLoader.html" target="viewer">TextureLoader</a></li></ul></div><div><h3>加载器 / 管理器</h3><ul><li><a href="api/zh/loaders/managers/DefaultLoadingManager.html" target="viewer">DefaultLoadingManager</a></li><li><a href="api/zh/loaders/managers/LoadingManager.html" target="viewer">LoadingManager</a></li></ul></div><div><h3>材质</h3><ul><li><a href="api/zh/materials/LineBasicMaterial.html" target="viewer">LineBasicMaterial</a></li><li><a href="api/zh/materials/LineDashedMaterial.html" target="viewer">LineDashedMaterial</a></li><li><a href="api/zh/materials/Material.html" target="viewer">Material</a></li><li><a href="api/zh/materials/MeshBasicMaterial.html" target="viewer">MeshBasicMaterial</a></li><li><a href="api/zh/materials/MeshDepthMaterial.html" target="viewer">MeshDepthMaterial</a></li><li><a href="api/zh/materials/MeshDistanceMaterial.html" target="viewer">MeshDistanceMaterial</a></li><li><a href="api/zh/materials/MeshLambertMaterial.html" target="viewer">MeshLambertMaterial</a></li><li><a href="api/zh/materials/MeshMatcapMaterial.html" target="viewer">MeshMatcapMaterial</a></li><li><a href="api/zh/materials/MeshNormalMaterial.html" target="viewer">MeshNormalMaterial</a></li><li><a href="api/zh/materials/MeshPhongMaterial.html" target="viewer">MeshPhongMaterial</a></li><li><a href="api/zh/materials/MeshPhysicalMaterial.html" target="viewer">MeshPhysicalMaterial</a></li><li><a href="api/zh/materials/MeshStandardMaterial.html" target="viewer">MeshStandardMaterial</a></li><li><a href="api/zh/materials/MeshToonMaterial.html" target="viewer">MeshToonMaterial</a></li><li><a href="api/zh/materials/PointsMaterial.html" target="viewer">PointsMaterial</a></li><li><a href="api/zh/materials/RawShaderMaterial.html" target="viewer">RawShaderMaterial</a></li><li><a href="api/zh/materials/ShaderMaterial.html" target="viewer">ShaderMaterial</a></li><li><a href="api/zh/materials/ShadowMaterial.html" target="viewer">ShadowMaterial</a></li><li><a href="api/zh/materials/SpriteMaterial.html" target="viewer">SpriteMaterial</a></li></ul></div><div><h3>数学库</h3><ul><li><a href="api/zh/math/Box2.html" target="viewer">Box2</a></li><li><a href="api/zh/math/Box3.html" target="viewer">Box3</a></li><li><a href="api/zh/math/Color.html" target="viewer">Color</a></li><li><a href="api/zh/math/Cylindrical.html" target="viewer">Cylindrical</a></li><li><a href="api/zh/math/Euler.html" target="viewer">Euler</a></li><li><a href="api/zh/math/Frustum.html" target="viewer">Frustum</a></li><li><a href="api/zh/math/Interpolant.html" target="viewer">Interpolant</a></li><li><a href="api/zh/math/Line3.html" target="viewer">Line3</a></li><li><a href="api/zh/math/MathUtils.html" target="viewer">MathUtils</a></li><li><a href="api/zh/math/Matrix3.html" target="viewer">Matrix3</a></li><li><a href="api/zh/math/Matrix4.html" target="viewer">Matrix4</a></li><li><a href="api/zh/math/Plane.html" target="viewer">Plane</a></li><li><a href="api/zh/math/Quaternion.html" target="viewer">Quaternion</a></li><li><a href="api/zh/math/Ray.html" target="viewer">Ray</a></li><li><a href="api/zh/math/Sphere.html" target="viewer">Sphere</a></li><li><a href="api/zh/math/Spherical.html" target="viewer">Spherical</a></li><li><a href="api/zh/math/SphericalHarmonics3.html" target="viewer">SphericalHarmonics3</a></li><li><a href="api/zh/math/Triangle.html" target="viewer">Triangle</a></li><li><a href="api/zh/math/Vector2.html" target="viewer">Vector2</a></li><li><a href="api/zh/math/Vector3.html" target="viewer">Vector3</a></li><li><a href="api/zh/math/Vector4.html" target="viewer">Vector4</a></li></ul></div><div><h3>数学库 / 插值</h3><ul><li><a href="api/zh/math/interpolants/CubicInterpolant.html" target="viewer">CubicInterpolant</a></li><li><a href="api/zh/math/interpolants/DiscreteInterpolant.html" target="viewer">DiscreteInterpolant</a></li><li><a href="api/zh/math/interpolants/LinearInterpolant.html" target="viewer">LinearInterpolant</a></li><li><a href="api/zh/math/interpolants/QuaternionLinearInterpolant.html" target="viewer">QuaternionLinearInterpolant</a></li></ul></div><div><h3>物体</h3><ul><li><a href="api/zh/objects/Bone.html" target="viewer">Bone</a></li><li><a href="api/zh/objects/Group.html" target="viewer">Group</a></li><li><a href="api/zh/objects/InstancedMesh.html" target="viewer">InstancedMesh</a></li><li><a href="api/zh/objects/Line.html" target="viewer">Line</a></li><li><a href="api/zh/objects/LineLoop.html" target="viewer">LineLoop</a></li><li><a href="api/zh/objects/LineSegments.html" target="viewer">LineSegments</a></li><li><a href="api/zh/objects/LOD.html" target="viewer">LOD</a></li><li><a href="api/zh/objects/Mesh.html" target="viewer">Mesh</a></li><li><a href="api/zh/objects/Points.html" target="viewer">Points</a></li><li><a href="api/zh/objects/Skeleton.html" target="viewer">Skeleton</a></li><li><a href="api/zh/objects/SkinnedMesh.html" target="viewer">SkinnedMesh</a></li><li><a href="api/zh/objects/Sprite.html" target="viewer">Sprite</a></li></ul></div><div><h3>渲染器</h3><ul><li><a href="api/zh/renderers/WebGL3DRenderTarget.html" target="viewer">WebGL3DRenderTarget</a></li><li><a href="api/zh/renderers/WebGLArrayRenderTarget.html" target="viewer">WebGLArrayRenderTarget</a></li><li><a href="api/zh/renderers/WebGLMultipleRenderTargets.html" target="viewer">WebGLMultipleRenderTargets</a></li><li><a href="api/zh/renderers/WebGLRenderer.html" target="viewer">WebGLRenderer</a></li><li><a href="api/zh/renderers/WebGL1Renderer.html" target="viewer">WebGL1Renderer</a></li><li><a href="api/zh/renderers/WebGLRenderTarget.html" target="viewer">WebGLRenderTarget</a></li><li><a href="api/zh/renderers/WebGLCubeRenderTarget.html" target="viewer">WebGLCubeRenderTarget</a></li></ul></div><div><h3>渲染器 / 着色器</h3><ul><li><a href="api/zh/renderers/shaders/ShaderChunk.html" target="viewer">ShaderChunk</a></li><li><a href="api/zh/renderers/shaders/ShaderLib.html" target="viewer">ShaderLib</a></li><li><a href="api/zh/renderers/shaders/UniformsLib.html" target="viewer">UniformsLib</a></li><li><a href="api/zh/renderers/shaders/UniformsUtils.html" target="viewer">UniformsUtils</a></li></ul></div><div><h3>渲染器 / WebXR</h3><ul><li><a href="api/zh/renderers/webxr/WebXRManager.html" target="viewer">WebXRManager</a></li></ul></div><div><h3>场景</h3><ul><li><a href="api/zh/scenes/Fog.html" target="viewer">Fog</a></li><li><a href="api/zh/scenes/FogExp2.html" target="viewer">FogExp2</a></li><li><a href="api/zh/scenes/Scene.html" target="viewer">Scene</a></li></ul></div><div><h3>纹理贴图</h3><ul><li><a href="api/zh/textures/CanvasTexture.html" target="viewer">CanvasTexture</a></li><li><a href="api/zh/textures/CompressedTexture.html" target="viewer">CompressedTexture</a></li><li><a href="api/zh/textures/CompressedArrayTexture.html" target="viewer">CompressedArrayTexture</a></li><li><a href="api/zh/textures/CubeTexture.html" target="viewer">CubeTexture</a></li><li><a href="api/zh/textures/DataArrayTexture.html" target="viewer">DataArrayTexture</a></li><li><a href="api/zh/textures/Data3DTexture.html" target="viewer">Data3DTexture</a></li><li><a href="api/zh/textures/DataTexture.html" target="viewer">DataTexture</a></li><li><a href="api/zh/textures/DepthTexture.html" target="viewer">DepthTexture</a></li><li><a href="api/zh/textures/FramebufferTexture.html" target="viewer">FramebufferTexture</a></li><li><a href="api/zh/textures/Source.html" target="viewer">Source</a></li><li><a href="api/zh/textures/Texture.html" target="viewer">Texture</a></li><li><a href="api/zh/textures/VideoTexture.html" target="viewer">VideoTexture</a></li></ul></div><h2>Addons</h2><div><h3>控制</h3><ul><li><a href="examples/zh/controls/DragControls.html" target="viewer">DragControls</a></li><li><a href="examples/zh/controls/FirstPersonControls.html" target="viewer">FirstPersonControls</a></li><li><a href="examples/zh/controls/FlyControls.html" target="viewer">FlyControls</a></li><li><a href="examples/zh/controls/OrbitControls.html" target="viewer">OrbitControls</a></li><li><a href="examples/zh/controls/PointerLockControls.html" target="viewer">PointerLockControls</a></li><li><a href="examples/zh/controls/TrackballControls.html" target="viewer">TrackballControls</a></li><li><a href="examples/zh/controls/TransformControls.html" target="viewer">TransformControls</a></li></ul></div><div><h3>几何体</h3><ul><li><a href="examples/zh/geometries/ConvexGeometry.html" target="viewer">ConvexGeometry</a></li><li><a href="examples/zh/geometries/DecalGeometry.html" target="viewer">DecalGeometry</a></li><li><a href="examples/zh/geometries/ParametricGeometry.html" target="viewer">ParametricGeometry</a></li><li><a href="examples/zh/geometries/TextGeometry.html" target="viewer">TextGeometry</a></li></ul></div><div><h3>辅助对象</h3><ul><li><a href="examples/zh/helpers/LightProbeHelper.html" target="viewer">LightProbeHelper</a></li><li><a href="examples/zh/helpers/PositionalAudioHelper.html" target="viewer">PositionalAudioHelper</a></li><li><a href="examples/zh/helpers/RectAreaLightHelper.html" target="viewer">RectAreaLightHelper</a></li><li><a href="examples/zh/helpers/VertexNormalsHelper.html" target="viewer">VertexNormalsHelper</a></li></ul></div><div><h3>灯光</h3><ul><li><a href="examples/zh/lights/LightProbeGenerator.html" target="viewer">LightProbeGenerator</a></li></ul></div><div><h3>加载器</h3><ul><li><a href="examples/zh/loaders/FontLoader.html" target="viewer">FontLoader</a></li><li><a href="examples/zh/loaders/DracoLoader.html" target="viewer">DracoLoader</a></li><li><a href="examples/zh/loaders/GLTFLoader.html" target="viewer">GLTFLoader</a></li><li><a href="examples/zh/loaders/MMDLoader.html" target="viewer">MMDLoader</a></li><li><a href="examples/zh/loaders/MTLLoader.html" target="viewer">MTLLoader</a></li><li><a href="examples/zh/loaders/OBJLoader.html" target="viewer">OBJLoader</a></li><li><a href="examples/zh/loaders/PCDLoader.html" target="viewer">PCDLoader</a></li><li><a href="examples/zh/loaders/SVGLoader.html" target="viewer">SVGLoader</a></li><li><a href="examples/zh/loaders/TGALoader.html" target="viewer">TGALoader</a></li></ul></div><div><h3>物体</h3><ul><li><a href="examples/zh/objects/Lensflare.html" target="viewer">Lensflare</a></li></ul></div><div><h3>后期处理</h3><ul><li><a href="examples/zh/postprocessing/EffectComposer.html" target="viewer">EffectComposer</a></li></ul></div><div><h3>渲染器</h3><ul><li><a href="examples/zh/renderers/CSS2DRenderer.html" target="viewer">CSS2DRenderer</a></li><li><a href="examples/zh/renderers/CSS3DRenderer.html" target="viewer">CSS3DRenderer</a></li><li><a href="examples/zh/renderers/SVGRenderer.html" target="viewer">SVGRenderer</a></li></ul></div><div><h3>实用工具</h3><ul><li><a href="examples/zh/utils/BufferGeometryUtils.html" target="viewer">BufferGeometryUtils</a></li><li><a href="examples/zh/utils/SceneUtils.html" target="viewer">SceneUtils</a></li><li><a href="examples/zh/utils/SkeletonUtils.html" target="viewer">SkeletonUtils</a></li></ul></div><h2>开发者参考</h2><div><h3>WebGL渲染器</h3><ul><li><a href="api/zh/renderers/webgl/WebGLProgram.html" target="viewer">WebGLProgram</a></li></ul></div></div>
            """
            
            # 处理HTML内容
            doc_links = scraper.process_html_content(animation_html, "animation")
            
            # 爬取这些链接
            all_docs = []
            with ThreadPoolExecutor(max_workers=scraper.max_workers) as executor:
                future_to_doc = {executor.submit(scraper.scrape_single_doc, doc): doc for doc in doc_links}
                
                for future in as_completed(future_to_doc):
                    doc_data = future.result()
                    if doc_data:
                        all_docs.append(doc_data)
                    
                    # 避免请求过于频繁
                    time.sleep(0.5)
            
            # 保存结果
            os.makedirs("threejs_docs_output", exist_ok=True)
            with open("threejs_docs_output/animation.json", 'w', encoding='utf-8') as f:
                json.dump(all_docs, f, ensure_ascii=False, indent=2)
            
            print(f"从HTML内容中爬取完成，共获取 {len(all_docs)} 个文档")
            docs = all_docs
        else:
            print(f"爬取完成，共获取 {len(docs)} 个文档")
        
        # 显示一些统计信息
        categories = {}
        for doc in docs:
            cat = doc['category']
            categories[cat] = categories.get(cat, 0) + 1
        
        print("\n分类统计:")
        for cat, count in sorted(categories.items()):
            print(f"  {cat}: {count} 个文档")
            
    except KeyboardInterrupt:
        print("\n用户中断爬取")
    except Exception as e:
        print(f"爬取过程中出现错误: {str(e)}")

if __name__ == "__main__":
    main()
